{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "actor无通信决策,但是公用一个critic.\n",
    "\n",
    "这意味着训练时有通信,测试时无通信"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEYCAYAAABlUvL1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnOklEQVR4nO3deVgUZ54H8G/1ySHdzSHdEkHJxKhEvMBgr+aYyEoMk0uSjT5EXR83rgqOV7IJ+xjdzWPEdWaeZM1Gncw8o+5MjBN2xowajyGYEKMtKsYsoqLGAzy6iSLdgFzd/e4fLjXpqJHWhqLh+3meeh6p9+2qX5XUlzq7JCGEABFRJ1MpXQAR9UwMHyJSBMOHiBTB8CEiRTB8iEgRDB8iUgTDh4gUwfAhIkUwfIhIEQwfIlKEYuHz/vvvo3///ggJCUFaWhoOHDigVClEpABFwuePf/wjFi5ciKVLl+Lw4cMYNmwYMjIyUF1drUQ5RKQASYkHS9PS0jBq1Cj813/9FwDA6/UiPj4ec+fOxRtvvNHZ5RCRAjSdPcOWlhaUlpYiLy9PHqdSqZCeng6bzXbLzzQ3N6O5uVn+2ev1oqamBtHR0ZAkqcNrJqL2EUKgrq4OcXFxUKl+/MCq08PnypUr8Hg8MJvNPuPNZjNOnDhxy8/k5+fj3//93zujPCIKgKqqKvTt2/dH+3R6+NyNvLw8LFy4UP7Z6XQiISEBVVVVMBgMClZGRN/ncrkQHx+PiIiIO/bt9PCJiYmBWq2Gw+HwGe9wOGCxWG75Gb1eD71ef9N4g8HA8CHqgtpzOqTTr3bpdDqkpKSgqKhIHuf1elFUVASr1drZ5RCRQhQ57Fq4cCGmTZuG1NRUPPzww3j33XfR0NCA6dOnK1EOESlAkfB56aWX8N1332HJkiWw2+0YPnw4du7cedNJaCLqvhS5z+deuVwuGI1GOJ1OnvMh6kL82Tb5bBcRKYLhQ0SKYPgQkSIYPkSkCIYPESmC4UNEimD4EJEiGD5EpAiGDxEpguFDRIpg+BCRIhg+RKQIhg8RKYLhQ0SKYPgQkSIYPkSkCIYPESmC4UNEimD4EJEiGD5EpAiGDxEpguFDRIrwO3y+/PJLPP3004iLi4MkSfjkk0982oUQWLJkCfr06YPQ0FCkp6fj1KlTPn1qamqQnZ0Ng8EAk8mEGTNmoL6+/p4WhIiCi9/h09DQgGHDhuH999+/ZfvKlSuxatUqrF27FiUlJQgPD0dGRgaamprkPtnZ2SgvL0dhYSG2bduGL7/8EjNnzrz7pSCi4CPuAQCxefNm+Wev1yssFov4xS9+IY+rra0Ver1efPTRR0IIIY4dOyYAiIMHD8p9duzYISRJEhcvXmzXfJ1OpwAgnE7nvZRPRAHmz7YZ0HM+Z8+ehd1uR3p6ujzOaDQiLS0NNpsNAGCz2WAymZCamir3SU9Ph0qlQklJyS2n29zcDJfL5TMQUXALaPjY7XYAuOmd62azWW6z2+2IjY31addoNIiKipL7/FB+fj6MRqM8xMfHB7JsIlJAUFztysvLg9PplIeqqiqlSyKiexTQ8LFYLAAAh8PhM97hcMhtFosF1dXVPu1utxs1NTVynx/S6/UwGAw+AxEFt4CGT2JiIiwWC4qKiuRxLpcLJSUlsFqtAACr1Yra2lqUlpbKfXbv3g2v14u0tLRAlkNEXZjG3w/U19fj9OnT8s9nz57FkSNHEBUVhYSEBMyfPx/Lli3DgAEDkJiYiDfffBNxcXF47rnnAACDBw/Gk08+iVdeeQVr165Fa2srcnNzMWnSJMTFxQVswYioi/P3Utrnn38uANw0TJs2TQhx43L7m2++Kcxms9Dr9WLcuHGioqLCZxpXr14VkydPFr169RIGg0FMnz5d1NXVtbsGXmon6pr82TYlIYRQMPvuisvlgtFohNPp5Pkfoi7En20zKK52EVH3w/AhIkUwfIhIEQwfIlIEw4eIFMHwISJFMHyISBEMHyJSBMOHiBTB8CEiRTB8iEgRDB8iUgTDh4gUwfAhIkUwfIhIEQwfIlIEw4eIFMHwISJFMHyISBEMHyJSBMOHiBTB8CEiRfgVPvn5+Rg1ahQiIiIQGxuL5557DhUVFT59mpqakJOTg+joaPTq1QtZWVk3vT65srISmZmZCAsLQ2xsLF577TW43e57XxoiChp+hU9xcTFycnKwf/9+FBYWorW1FePHj0dDQ4PcZ8GCBdi6dSsKCgpQXFyMS5cuYeLEiXK7x+NBZmYmWlpasG/fPmzYsAHr16/HkiVLArdURNT13cvbCaurqwUAUVxcLIQQora2Vmi1WlFQUCD3OX78uAAgbDabEEKI7du3C5VKJex2u9xnzZo1wmAwiObm5nbNl28sJeqa/Nk27+mcj9PpBABERUUBAEpLS9Ha2or09HS5z6BBg5CQkACbzQYAsNlsSE5OhtlslvtkZGTA5XKhvLz8lvNpbm6Gy+XyGYgouN11+Hi9XsyfPx9jxozBkCFDAAB2ux06nQ4mk8mnr9lsht1ul/t8P3ja2tvabiU/Px9Go1Ee4uPj77ZsIuoi7jp8cnJycPToUWzatCmQ9dxSXl4enE6nPFRVVXX4PImoY2nu5kO5ubnYtm0bvvzyS/Tt21ceb7FY0NLSgtraWp+9H4fDAYvFIvc5cOCAz/Taroa19fkhvV4PvV5/N6USURfl156PEAK5ubnYvHkzdu/ejcTERJ/2lJQUaLVaFBUVyeMqKipQWVkJq9UKALBarSgrK0N1dbXcp7CwEAaDAUlJSfeyLEQURPza88nJycHGjRvxl7/8BREREfI5GqPRiNDQUBiNRsyYMQMLFy5EVFQUDAYD5s6dC6vVitGjRwMAxo8fj6SkJEyZMgUrV66E3W7H4sWLkZOTw70bop7En8toAG45rFu3Tu7T2Ngo5syZIyIjI0VYWJh4/vnnxeXLl32mc+7cOTFhwgQRGhoqYmJixKJFi0Rra2u76+CldqKuyZ9tUxJCCOWi7+64XC4YjUY4nU4YDAalyyGi/+fPtslnu4hIEQwfIlIEw4eIFMHwISJFMHyISBEMHyJSBMOHiBTB8CEiRTB8iEgRDB8iUsRdfaVGMPB6vbh06RK2bNmCa9euoVevXnjmmWfQr18/qFTMXCKldcvwaWhowAcffIBPPvkELpcLkiRBCIGCggJMmDABubm5MBqNSpdJ1KN1u/BpamrC22+/jZ07dwKAvJcjSRKuX7+OP/3pT3A4HHj77bcRERGhZKlEPVq3Ov4QQqC4uBg7d+6EJEmQJMmnvW3cnj17sH37dgThA/1E3Ua3Cp+mpiZs2LDhjv0kScLGjRtRV1fXCVUR0a10q/Cprq7GyZMnb9rj+SFJknD+/HmcP3++kyojoh/qVuEDgIdSREGiW4VPZGQk7rvvvjsGkBACMTExt31bBhF1vG4VPhEREZg8eXK7wicrKwsxMTGdVBkR/VC3Ch9JkpCZmYnhw4ffNoCEEHjwwQfxwgsv3PHcEBF1nG4VPsCNvZ//+I//wJgxY+SbC4G/nQsaOXIkfvnLX3Kvh0hh3fLtFUIIuN1ufP311/joo4/kxyteeuklpKWlQavVcq+HqAP49WYZf97Js3r1apGcnCwiIiJERESEGD16tNi+fbvc3vbOrqioKBEeHi4mTpwo7Ha7zzTOnz8vnnrqKREaGip69+4tXn31Vb/e2SVE+98N5PV6bxqIqOP4894uvw67+vbtixUrVqC0tBSHDh3CE088gWeffRbl5eUAgAULFmDr1q0oKChAcXExLl26hIkTJ8qf93g8yMzMREtLC/bt24cNGzZg/fr1WLJkib8B2y5tdzR/fyCiLuJeky4yMlL89re/FbW1tUKr1YqCggK57fjx4wKAsNlsQgghtm/fLlQqlc/e0Jo1a4TBYBDNzc3tniffWErUNXXYns/3eTwebNq0CQ0NDbBarSgtLUVrayvS09PlPoMGDUJCQgJsNhsAwGazITk5GWazWe6TkZEBl8sl7z3dSnNzM1wul89ARMHN7/ApKytDr169oNfrMWvWLGzevBlJSUmw2+3Q6XQwmUw+/c1mM+x2OwDAbrf7BE9be1vb7eTn58NoNMpDfHy8v2UTURfjd/gMHDgQR44cQUlJCWbPno1p06bh2LFjHVGbLC8vD06nUx6qqqo6dH5E1PH8/j4fnU6HBx54AACQkpKCgwcP4j//8z/x0ksvoaWlBbW1tT57Pw6HQ36MwWKx4MCBAz7Tczgcctvt6PV66PV6f0sloi7snm8y9Hq9aG5uRkpKCrRaLYqKiuS2iooKVFZWwmq1AgCsVivKyspQXV0t9yksLITBYEBSUtK9lkJEQcSvPZ+8vDxMmDABCQkJqKurw8aNG/HFF19g165dMBqNmDFjBhYuXIioqCgYDAbMnTsXVqsVo0ePBgCMHz8eSUlJmDJlClauXAm73Y7FixcjJyeHezZEPYxf4VNdXY2pU6fi8uXLMBqNGDp0KHbt2oW///u/BwC88847UKlUyMrKQnNzMzIyMrB69Wr582q1Gtu2bcPs2bNhtVoRHh6OadOm4a233grsUhFRl9ctH68gImX4s212uwdLiSg4MHyISBEMHyJSBMOHiBTB8CEiRTB8iEgRDB8iUgTDh4gUwfAhIkUwfIhIEQwfIlIEw4eIFMHwISJFMHyISBEMHyJSBMOHiBTB8CEiRTB8iEgRDB8iUgTDh4gU4fdLA4OFx+PBd999h6NHT+LMGQdOn/7b65hVKgmjRj0AszkKyckDYTAYIEmSgtUS9TzdKnyEELhy5Sr27fsaX3xxAsePN6KpyQC1OhRqte874ktKLgD4FibT5xgxIho//ekQpKYOQ2hoqDLFE/Uw93TYtWLFCkiShPnz58vjmpqakJOTg+joaPTq1QtZWVnyK5HbVFZWIjMzE2FhYYiNjcVrr70Gt9t9L6WgqakJBQV/xT//83q8++45lJfHwuvtD70+GhpNGCRJ8hl0OiN0ulg0NCRiz55wLF16GDk5a3HoUBk8Hs891UJEd3bXez4HDx7Er3/9awwdOtRn/IIFC/Dpp5+ioKAARqMRubm5mDhxIvbu3QvgxuFQZmYmLBYL9u3bh8uXL2Pq1KnQarVYvny533UIIfDtt+ewZs1f8c03KqhUCdDp2n8IdSOMNFCpeuPCBYF//dfP8cwzx/HyyxkwmYx+10M/TggBp9OJ1tZWaDQamEwmHvL2UHf10sD6+nqMHDkSq1evxrJlyzB8+HC8++67cDqd6N27NzZu3IgXXngBAHDixAkMHjwYNpsNo0ePxo4dO/Czn/0Mly5dgtl841Bo7dq1eP311/Hdd99Bp9Pdcf5tLyarqanB+fOXsGzZ53A6o6BSaf1dlJsIIeDxNGLYMDdef/1pxMb25sYRAEIIXL58GR9//DG2bt2K69evIyQkBJmZmZg0aRLuu+8+ruduoMNfGpiTk4PMzEykp6f7jC8tLUVra6vP+EGDBiEhIQE2mw0AYLPZkJycLAcPAGRkZMDlcqG8vPyW82tubobL5fIZAGDLli/w+uufweWKDUjwADf2hDSaMJSVhWDOnD/gxInTAZluTyaEwMGDBzF16lRs2LAB165dQ0tLC5xOJz788ENMmTIFX331FYLw5bl0D/wOn02bNuHw4cPIz8+/qc1ut0On08FkMvmMN5vNsNvtcp/vB09be1vbreTn58NoNMpDfHw8AODjj08BiOuQv5gqlQ4uVx+sXr0b1645Az79nuT8+fN44403UFNTA5VKJf9/SZIElUoFp9OJxYsX4/RpBn1P4lf4VFVVYd68efjwww8REhLSUTXdJC8vD06nUx6qqqoAAI2NsR06X5VKgxMneuGXv/wYbjdPQt8NIQQ++ugjXLt27bZ/JCRJgsvlwu9//3t4vd5OrpCU4lf4lJaWorq6GiNHjoRGo4FGo0FxcTFWrVoFjUYDs9mMlpYW1NbW+nzO4XDAYrEAACwWy01Xv9p+buvzQ3q9HgaDwWcA0CnnCFQqHQ4cEPjrX3lYcDfsdju2b98OlerHf9VUKhU+++wzXLhwoZMqI6X5FT7jxo1DWVkZjhw5Ig+pqanIzs6W/63ValFUVCR/pqKiApWVlbBarQAAq9WKsrIyVFdXy30KCwthMBiQlJQUoMUKLEky4be/PQy73XHnzuTj+vXraGhoCHhfCn5+XWqPiIjAkCFDfMaFh4cjOjpaHj9jxgwsXLgQUVFRMBgMmDt3LqxWK0aPHg0AGD9+PJKSkjBlyhSsXLkSdrsdixcvRk5ODvR6fYAWK7BuHBZE489//gJz5rzEqzJ+aNtDbs+9U219qWcI+LNd77zzDn72s58hKysLjz76KCwWC/785z/L7Wq1Gtu2bYNarYbVasXLL7+MqVOn4q233gp0KQGlVodi794qNDY2Kl1KULnvvvtgtVrveMgqhEBKSgr69+/fOYWR4u7qPh+ltd1L8MQT66HRhHXafN3uS/jVr57A8OHJnTbP7uDw4cOYNWsW3G73LfcahRBQqVR477335MNzCk4dfp9PzxWF4uIynnj207BhwzBv3jyo1eqb1l1b8MyZMwejRo1SqEJSAg+w/aDRhODkySqlywg6arUakyZNQkJCAtatW4cjR47A7XZDrVZj6NChmDZtGh599FGe7+lh+L/tp6oqN65du4aoqCilSwkqarUajzzyCB5++GGUl5ejoaEBoaGhGDJkSKfeM0ZdB8PHT9eve9Ha2qp0GUFLr9dj5MiRSpdBXQDP+RCRIhg+RKQIho+f9HqJJ0aJAoDh46d+/bQ82UwUAAwfP3g8zbj//hilyyDqFhg+fhDiGh57LJnPdhEFAMPHDzExTRg8eIDSZRB1CwyfdvJ4mpCW1gfh4eFKl0LULTB82kEIgbCwK3jhhcd5yEUUIAyfdvB6XZg27SHEx9+ndClE3UZQh09nPF3u9boxbFgrnn76p9zrIQqgoA4fne5Kh07f6/UgMbEWr7/+IrRa3lhIFEhBHT7PPtsXHk91h+wBeb1uhIRcxuzZj6B372ju9RAFWFCHz0svjce//VsawsKuwOsN3KttPJ5mPPBAHd57LwvDhz/E4CHqAEF9LKHRaDBmTAoMhnCsXr0bp0+HQq2OuOvpCSHg9X6HJ56IwD/900Tu8RB1oKD+Due274kVQqChoQGbNn2GTz75Fo2N0VCrw9odHEJ44HbXITbWhVmzxmLMmJHQagPz+mWinsSf73DuFuHTRgiBCxcuYvfug9iz5wzOnQO83kioVDqo1Xqffh5PI4RoQUjINSQlheHxxwfj0UdHISIigns7RHepx4ZPGyEE3G43Kisv4PDhYzh37iq+/fY7uV2SJKSmJsBiiUJq6hDExvaWxxPR3fMnfCD8sHTpUgHAZxg4cKDc3tjYKObMmSOioqJEeHi4mDhxorDb7T7TOH/+vHjqqadEaGio6N27t3j11VdFa2urP2UIp9MpAAin09mu/l6v95YDEQWWP9um3yecH3roIXz22Wfyz9//Yq0FCxbg008/RUFBAYxGI3JzczFx4kTs3bsXAODxeJCZmQmLxYJ9+/bh8uXLmDp1KrRaLZYvX+5vKe3GPRqiLsifVFu6dKkYNmzYLdtqa2uFVqsVBQUF8rjjx48LAMJmswkhhNi+fbtQqVQ+e0Nr1qwRBoNBNDc3t7sOf/d8iKhz+LNt+n2fz6lTpxAXF4f7778f2dnZqKysBACUlpaitbUV6enpct9BgwYhISEBNpsNAGCz2ZCcnAyz2Sz3ycjIgMvlQnl5+W3n2dzcDJfL5TMQUXDzK3zS0tKwfv167Ny5E2vWrMHZs2fxyCOPoK6uDna7HTqdDiaTyeczZrMZdrsdAGC3232Cp629re128vPzYTQa5SE+Pt6fsomoC/LrnM+ECRPkfw8dOhRpaWno168fPv74Y4SGhga8uDZ5eXlYuHCh/LPL5WIAEQW5e3q8wmQy4cEHH8Tp06dhsVjQ0tKC2tpanz4OhwMWiwUAYLFY4HA4bmpva7sdvV4Pg8HgMxBRcLun8Kmvr8e3336LPn36ICUlBVqtFkVFRXJ7RUUFKisrYbVaAQBWqxVlZWWorq6W+xQWFsJgMCApKeleSiGiIOPXYderr76Kp59+Gv369cOlS5ewdOlSqNVqTJ48GUajETNmzMDChQsRFRUFg8GAuXPnwmq1YvTo0QCA8ePHIykpCVOmTMHKlStht9uxePFi5OTkQK/X32HuRNSd+BU+Fy5cwOTJk3H16lX07t0bY8eOxf79+9G79407hN955x2oVCpkZWWhubkZGRkZWL16tfx5tVqNbdu2Yfbs2bBarQgPD8e0adPw1ltvBXapiKjL65aPVxCRMvzZNoP6+3yIKHgxfIhIEQwfIlIEw4eIFMHwISJFMHyISBEMHyJSBMOHiBTB8CEiRTB8iEgRDB8iUgTDh4gUwfAhIkUwfIhIEQwfIlIEw4eIFMHwISJFMHyISBEMHyJShF9fIE/dk9vtht1ejRMnTuPkyUs4f/6K3KbRqDF69ACYzVFISnoQ4eHhkCRJwWqpu2D49FBCCDgc1di792sUF59ERUUzmpsjoFaHQa2O/X5P7N17BirVccTEfIbUVDMefzwZw4cPgU6nU6x+Cn58e0UPdP36dXz88WfYvPkkXC4TtNob6/BOezRCCHi9rfB6azBokEBu7gQMGvQTqFQ8eqcbOvTtFRcvXsTLL7+M6OhohIaGIjk5GYcOHZLbhRBYsmQJ+vTpg9DQUKSnp+PUqVM+06ipqUF2djYMBgNMJhNmzJiB+vp6f0shPwkhcOzYSbz22nr84Q9X0NiYAJ3OCEmS2nUoJUkS1GodtFoLTp82Y9GiT/Hf/70NDQ3XO6F66m78Cp9r165hzJgx0Gq12LFjB44dO4Zf/epXiIyMlPusXLkSq1atwtq1a1FSUoLw8HBkZGSgqalJ7pOdnY3y8nIUFhZi27Zt+PLLLzFz5szALRXdxO12Y//+I8jL+ytOnoyEWh1xT+duJEkFjycOv//9Faxc+SfU1FwLYLXUE/h12PXGG29g79692LNnzy3bhRCIi4vDokWL8OqrrwIAnE4nzGYz1q9fj0mTJuH48eNISkrCwYMHkZqaCgDYuXMnnnrqKVy4cAFxcXF3rIOHXf4RQmDduq3YtOk8AHPATxh7vU3o3fsKVqx4EQkJfQM6bQouHXbYtWXLFqSmpuLFF19EbGwsRowYgd/85jdy+9mzZ2G325Geni6PMxqNSEtLg81mAwDYbDaYTCY5eAAgPT0dKpUKJSUlt5xvc3MzXC6Xz0DtI4TAoUNH8ac/VaIjggcAVKoQVFfHYu3aQjQ0NAR8+tQ9+RU+Z86cwZo1azBgwADs2rULs2fPxs9//nNs2LABAGC32wEAZrPZ53Nms1lus9vtiI2N9WnXaDSIioqS+/xQfn4+jEajPMTHx/tTdo8lhEBl5QUsW1aE1tbYDr1ErlbrcPCgBqtXb4bX6+2w+VD34Vf4eL1ejBw5EsuXL8eIESMwc+ZMvPLKK1i7dm1H1QcAyMvLg9PplIeqqqoOnV934Xa78d57n+L69Y4NnjaSpEdhoQv79x9GEF5EpU7mV/j06dMHSUlJPuMGDx6MyspKAIDFYgEAOBwOnz4Oh0Nus1gsqK6u9ml3u92oqamR+/yQXq+HwWDwGejHCSGwZcvn+OYbDVQqbafM80bAReP997+C0+nslHlS8PIrfMaMGYOKigqfcSdPnkS/fv0AAImJibBYLCgqKpLbXS4XSkpKYLVaAQBWqxW1tbUoLS2V++zevRterxdpaWl3vSDkq66uHv/zP2WQpM4NaklSwW6PwI4dX3XqfCn4+BU+CxYswP79+7F8+XKcPn0aGzduxAcffICcnBwAN/7yzZ8/H8uWLcOWLVtQVlaGqVOnIi4uDs899xyAG3tKTz75JF555RUcOHAAe/fuRW5uLiZNmtSuK13UPuXlFXA49Io8CqFWR+CLL06hpaWl0+dNwcOvxytGjRqFzZs3Iy8vD2+99RYSExPx7rvvIjs7W+7zL//yL2hoaMDMmTNRW1uLsWPHYufOnQgJCZH7fPjhh8jNzcW4ceOgUqmQlZWFVatWBW6pejghBIqLj0Klirxz5w4gSRLOnvWiquoSfvKT/orUQF0fH6/ohlwuF2bM+DVcrv6K1dDaWo/p06MwZcqzitVAna9DH6+grq+m5hquXlW2Bo0mHCdOXORVL7othk83VFZ2CoBJ6TJw+nQ9GhsblS6DuiiGTzdUW1sHSVL26y4kSUJ9vRsej0fROqjrYvgQkSIYPkSkCIZPNxQeHgoh3IrWIIRASIiaXzRGt8XfjG5oxIhBAJT/fp2BAyMQFhamdBnURTF8uiGTyQijUdlL3B5PI+6/v2O+woO6B4ZPN2QymTB8eCS83lbFalCpajB27FDF5k9dH8OnG5IkCY899hA8nlpF5i+EQHy8QGJigiLzp+DA8Ommhg0bhMjI64rcYezxXMfYsf2h1+s7fd4UPBg+3ZTJZMKzzz4Ir7dzv9ZUCIHIyBo8/fSjnTpfCj4Mn25KkiT8wz+Mx4MP1kOIzrnL+MZ7vWowa1YaYmKiO2WeFLwYPt2YXq/HvHkToNE4OuXwSwg3xo7V4vHH03iVi+6I4dONSZKEBx/8CRYtGg21umMfc/d6W/HQQw2YNy8LGg3fwk13xvDp5iRJwk9/OgoZGVHweK50yB6Q19uKXr3smDPnCURGmgI+feqeGD49gEqlwpw5z2PRoiHQaq9CiMC82kYIAY+nCcnJjXjvvUkYMOD+gEyXegbuH/cQOp0OTz45FpGR4Vi79itcuGCEWn33jz4IISCEA5mZ0Zg+/QWYTMYAVks9AcOnB5EkCWlpIzBwYCLWr9+JXbsuoLU1Gmp1SLtPEHu9bng8dYiLq0du7k8xatRQqNXqDq6cuiN+h3MP5fV6cebMORQVHcK+fZW4cEENIUxQq/U+7/m6cWjVCCGaER5ei4ceisDjjw/GI4+MQlhYGK9qkQ9/tk2GTw8nhEBLSwvOnDmHI0cqcPr0d6iq+tsT8Wq1CqNG9UNcXDRSUh5CTEwMADB06Jb82TZ52NXDSZIEvV6PwYMHYvDggbe9GsawoUBj+JAPhgx1Fl5qJyJFMHyISBFBedjVdl7C5XIpXAkRfV/bNtme61hBGT5X//91nPHx8QpXQkS3UldXB6Pxx288DcrwiYqKAgBUVlbecQG7M5fLhfj4eFRVVfXoWw64Hm7oCutBCIG6ujrExcXdsW9Qhk/b61iMRmOP/mVrYzAYuB7A9dBG6fXQ3h0CnnAmIkUwfIhIEUEZPnq9HkuXLu3xX1DO9XAD18MNwbYegvLZLiIKfkG550NEwY/hQ0SKYPgQkSIYPkSkiKAMn/fffx/9+/dHSEgI0tLScODAAaVLCpj8/HyMGjUKERERiI2NxXPPPYeKigqfPk1NTcjJyUF0dDR69eqFrKwsOBwOnz6VlZXIzMxEWFgYYmNj8dprr8HtdnfmogTUihUrIEkS5s+fL4/rKevh4sWLePnllxEdHY3Q0FAkJyfj0KFDcrsQAkuWLEGfPn0QGhqK9PR0nDp1ymcaNTU1yM7OhsFggMlkwowZM1BfX9/Zi+JLBJlNmzYJnU4nfve734ny8nLxyiuvCJPJJBwOh9KlBURGRoZYt26dOHr0qDhy5Ih46qmnREJCgqivr5f7zJo1S8THx4uioiJx6NAhMXr0aPF3f/d3crvb7RZDhgwR6enp4uuvvxbbt28XMTExIi8vT4lFumcHDhwQ/fv3F0OHDhXz5s2Tx/eE9VBTUyP69esn/vEf/1GUlJSIM2fOiF27donTp0/LfVasWCGMRqP45JNPxDfffCOeeeYZkZiYKBobG+U+Tz75pBg2bJjYv3+/2LNnj3jggQfE5MmTlVgkWdCFz8MPPyxycnLknz0ej4iLixP5+fkKVtVxqqurBQBRXFwshBCitrZWaLVaUVBQIPc5fvy4ACBsNpsQQojt27cLlUol7Ha73GfNmjXCYDCI5ubmzl2Ae1RXVycGDBggCgsLxWOPPSaHT09ZD6+//roYO3bsbdu9Xq+wWCziF7/4hTyutrZW6PV68dFHHwkhhDh27JgAIA4ePCj32bFjh5AkSVy8eLHjir+DoDrsamlpQWlpKdLT0+VxKpUK6enpsNlsClbWcZxOJ4C/PUxbWlqK1tZWn3UwaNAgJCQkyOvAZrMhOTkZZrNZ7pORkQGXy4Xy8vJOrP7e5eTkIDMz02d5gZ6zHrZs2YLU1FS8+OKLiI2NxYgRI/Cb3/xGbj979izsdrvPejAajUhLS/NZDyaTCampqXKf9PR0qFQqlJSUdN7C/EBQhc+VK1fg8Xh8fpkAwGw2w263K1RVx/F6vZg/fz7GjBmDIUOGAADsdjt0Oh1MJpNP3++vA7vdfst11NYWLDZt2oTDhw8jPz//praesh7OnDmDNWvWYMCAAdi1axdmz56Nn//859iwYQOAvy3Hj20TdrsdsbGxPu0ajQZRUVGKroegfKq9p8jJycHRo0fx1VdfKV1Kp6uqqsK8efNQWFiIkJAQpctRjNfrRWpqKpYvXw4AGDFiBI4ePYq1a9di2rRpCld3b4JqzycmJgZqtfqmKxoOhwMWi0WhqjpGbm4utm3bhs8//xx9+/aVx1ssFrS0tKC2ttan//fXgcViueU6amsLBqWlpaiursbIkSOh0Wig0WhQXFyMVatWQaPRwGw294j10KdPHyQlJfmMGzx4MCorKwH8bTl+bJuwWCyorq72aXe73aipqVF0PQRV+Oh0OqSkpKCoqEge5/V6UVRUBKvVqmBlgSOEQG5uLjZv3ozdu3cjMTHRpz0lJQVardZnHVRUVKCyslJeB1arFWVlZT6/cIWFhTAYDDf9IndV48aNQ1lZGY4cOSIPqampyM7Olv/dE9bDmDFjbrrV4uTJk+jXrx8AIDExERaLxWc9uFwulJSU+KyH2tpalJaWyn12794Nr9eLtLS0TliK21DsVPdd2rRpk9Dr9WL9+vXi2LFjYubMmcJkMvlc0Qhms2fPFkajUXzxxRfi8uXL8nD9+nW5z6xZs0RCQoLYvXu3OHTokLBarcJqtcrtbZeYx48fL44cOSJ27twpevfuHVSXmG/l+1e7hOgZ6+HAgQNCo9GIt99+W5w6dUp8+OGHIiwsTPzhD3+Q+6xYsUKYTCbxl7/8Rfzv//6vePbZZ295qX3EiBGipKREfPXVV2LAgAG81H433nvvPZGQkCB0Op14+OGHxf79+5UuKWAA3HJYt26d3KexsVHMmTNHREZGirCwMPH888+Ly5cv+0zn3LlzYsKECSI0NFTExMSIRYsWidbW1k5emsD6Yfj0lPWwdetWMWTIEKHX68WgQYPEBx984NPu9XrFm2++Kcxms9Dr9WLcuHGioqLCp8/Vq1fF5MmTRa9evYTBYBDTp08XdXV1nbkYN+FXahCRIoLqnA8RdR8MHyJSBMOHiBTB8CEiRTB8iEgRDB8iUgTDh4gUwfAhIkUwfIhIEQwfIlIEw4eIFMHwISJF/B+KvmtWD60m1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        from pettingzoo.mpe import simple_spread_v3\n",
    "        self.N = 2\n",
    "        env = simple_spread_v3.env(N=self.N,\n",
    "                                   local_ratio=0.5,\n",
    "                                   max_cycles=1e8,\n",
    "                                   render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        self.env.reset()\n",
    "        self.step_n = 0\n",
    "\n",
    "        import numpy as np\n",
    "        mark0, mark1 = self.env.env.env.world.landmarks\n",
    "        dist = np.array(mark0.state.p_pos) - np.array(mark1.state.p_pos)\n",
    "        dist = (dist**2).sum()**0.5\n",
    "        if dist < 1:\n",
    "            return self.reset()\n",
    "\n",
    "        return self.state()\n",
    "\n",
    "    def state(self):\n",
    "        state = []\n",
    "        for i in self.env.agents:\n",
    "            state.append(env.observe(i).tolist())\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        reward_sum = [0] * self.N\n",
    "        for i in range(5):\n",
    "            if i != 0:\n",
    "                action = [-1, -1]\n",
    "            next_state, reward, over = self._step(action)\n",
    "            for j in range(self.N):\n",
    "                reward_sum[j] += reward[j]\n",
    "            self.step_n -= 1\n",
    "\n",
    "        self.step_n += 1\n",
    "\n",
    "        return next_state, reward_sum, over\n",
    "\n",
    "    def _step(self, action):\n",
    "        for i, _ in enumerate(env.agent_iter(self.N)):\n",
    "            self.env.step(action[i] + 1)\n",
    "\n",
    "        reward = [self.env.rewards[i] for i in self.env.agents]\n",
    "\n",
    "        _, _, termination, truncation, _ = env.last()\n",
    "        over = termination or truncation\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 50:\n",
    "            over = True\n",
    "\n",
    "        return self.state(), reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<__main__.A2C at 0x7f9cb02918e0>, <__main__.A2C at 0x7f9c3da39940>]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class A2C:\n",
    "\n",
    "    def __init__(self, model_actor, model_critic, model_critic_delay,\n",
    "                 optimizer_actor, optimizer_critic):\n",
    "        self.model_actor = model_actor\n",
    "        self.model_critic = model_critic\n",
    "        self.model_critic_delay = model_critic_delay\n",
    "        self.optimizer_actor = optimizer_actor\n",
    "        self.optimizer_critic = optimizer_critic\n",
    "\n",
    "        self.model_critic_delay.load_state_dict(self.model_critic.state_dict())\n",
    "        self.requires_grad(self.model_critic_delay, False)\n",
    "\n",
    "    def soft_update(self, _from, _to):\n",
    "        for _from, _to in zip(_from.parameters(), _to.parameters()):\n",
    "            value = _to.data * 0.99 + _from.data * 0.01\n",
    "            _to.data.copy_(value)\n",
    "\n",
    "    def requires_grad(self, model, value):\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad_(value)\n",
    "\n",
    "    def train_critic(self, state, reward, next_state, over):\n",
    "        self.requires_grad(self.model_critic, True)\n",
    "        self.requires_grad(self.model_actor, False)\n",
    "\n",
    "        #计算values和targets\n",
    "        value = self.model_critic(state)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            target = self.model_critic_delay(next_state)\n",
    "        target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "        #时序差分误差,也就是tdloss\n",
    "        loss = torch.nn.functional.mse_loss(value, target)\n",
    "\n",
    "        loss.backward()\n",
    "        self.optimizer_critic.step()\n",
    "        self.optimizer_critic.zero_grad()\n",
    "        self.soft_update(self.model_critic, self.model_critic_delay)\n",
    "\n",
    "        #减去value相当于去基线\n",
    "        return (target - value).detach()\n",
    "\n",
    "    def train_actor(self, state, action, value):\n",
    "        self.requires_grad(self.model_critic, False)\n",
    "        self.requires_grad(self.model_actor, True)\n",
    "\n",
    "        #重新计算动作的概率\n",
    "        prob = self.model_actor(state)\n",
    "        prob = prob.gather(dim=1, index=action)\n",
    "\n",
    "        #根据策略梯度算法的导函数实现\n",
    "        #函数中的Q(state,action),这里使用critic模型估算\n",
    "        prob = (prob + 1e-8).log() * value\n",
    "        loss = -prob.mean()\n",
    "\n",
    "        loss.backward()\n",
    "        self.optimizer_actor.step()\n",
    "        self.optimizer_actor.zero_grad()\n",
    "\n",
    "        return loss.item()\n",
    "\n",
    "\n",
    "model_actor = [\n",
    "    torch.nn.Sequential(\n",
    "        torch.nn.Linear(6 * env.N, 64),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.Linear(64, 64),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.Linear(64, 4),\n",
    "        torch.nn.Softmax(dim=1),\n",
    "    ) for _ in range(env.N)\n",
    "]\n",
    "\n",
    "model_critic, model_critic_delay = [\n",
    "    torch.nn.Sequential(\n",
    "        torch.nn.Linear(6 * env.N * env.N, 64),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.Linear(64, 64),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.Linear(64, 1),\n",
    "    ) for _ in range(2)\n",
    "]\n",
    "\n",
    "optimizer_actor = [\n",
    "    torch.optim.Adam(model_actor[i].parameters(), lr=1e-3)\n",
    "    for i in range(env.N)\n",
    "]\n",
    "optimizer_critic = torch.optim.Adam(model_critic.parameters(), lr=5e-3)\n",
    "\n",
    "a2c = [\n",
    "    A2C(model_actor[i], model_critic, model_critic_delay, optimizer_actor[i],\n",
    "        optimizer_critic) for i in range(env.N)\n",
    "]\n",
    "\n",
    "model_actor = None\n",
    "model_critic = None\n",
    "model_critic_delay = None\n",
    "optimizer_actor = None\n",
    "optimizer_critic = None\n",
    "\n",
    "a2c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-435.03021240234375"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    state = []\n",
    "    action = []\n",
    "    reward = []\n",
    "    next_state = []\n",
    "    over = []\n",
    "\n",
    "    s = env.reset()\n",
    "    o = False\n",
    "    while not o:\n",
    "        a = []\n",
    "        for i in range(env.N):\n",
    "            #计算动作\n",
    "            prob = a2c[i].model_actor(torch.FloatTensor(s[i]).reshape(\n",
    "                1, -1))[0].tolist()\n",
    "            a.append(random.choices(range(4), weights=prob, k=1)[0])\n",
    "\n",
    "        #执行动作\n",
    "        ns, r, o = env.step(a)\n",
    "\n",
    "        state.append(s)\n",
    "        action.append(a)\n",
    "        reward.append(r)\n",
    "        next_state.append(ns)\n",
    "        over.append(o)\n",
    "\n",
    "        s = ns\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    state = torch.FloatTensor(state)\n",
    "    action = torch.LongTensor(action).unsqueeze(-1)\n",
    "    reward = torch.FloatTensor(reward).unsqueeze(-1)\n",
    "    next_state = torch.FloatTensor(next_state)\n",
    "    over = torch.LongTensor(over).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over, reward.sum().item()\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over, reward_sum = play()\n",
    "\n",
    "reward_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -11.440613746643066 -487.2028579711914\n",
      "2500 -1.119187593460083 -271.2435604095459\n",
      "5000 6.826124668121338 -253.62119140625\n",
      "7500 0.8457289934158325 -203.59983520507814\n",
      "10000 -1.034010648727417 -185.20421142578124\n",
      "12500 -3.779625177383423 -122.73852005004883\n",
      "15000 0.11820918321609497 -90.8776159286499\n",
      "17500 0.06188208609819412 -96.55424728393555\n",
      "20000 1.145486831665039 -98.22404899597169\n",
      "22500 -0.15374360978603363 -91.68431587219239\n",
      "25000 -0.2972413897514343 -83.5183292388916\n",
      "27500 -0.002570606069639325 -97.23491764068604\n",
      "30000 -0.031296394765377045 -110.00625648498536\n",
      "32500 -0.007811045274138451 -87.30725765228271\n",
      "35000 0.32335254549980164 -67.55157518386841\n",
      "37500 -0.10166460275650024 -104.05087776184082\n",
      "40000 -0.0728297010064125 -82.80452728271484\n",
      "42500 -0.23787322640419006 -84.73376636505127\n",
      "45000 -0.6099832653999329 -92.28320369720458\n",
      "47500 -0.054144881665706635 -84.60027885437012\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    #训练N局\n",
    "    for epoch in range(5_0000):\n",
    "        state, action, reward, next_state, over, _ = play()\n",
    "\n",
    "        #合并部分字段\n",
    "        state_c = state.flatten(start_dim=1)\n",
    "        reward_c = reward.sum(dim=1)\n",
    "        next_state_c = next_state.flatten(start_dim=1)\n",
    "\n",
    "        for i in range(env.N):\n",
    "            value = a2c[i].train_critic(state_c, reward_c, next_state_c, over)\n",
    "            loss = a2c[i].train_actor(state[:, i], action[:, i], value)\n",
    "\n",
    "        if epoch % 2500 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, loss, test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEYCAYAAABlUvL1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAmA0lEQVR4nO3de3QT150H8O9IsoRfkvBLsgsGt3EwLuYRHIwKCQm4GOI8SFy24VBKszQ01BDACU19TuKUtIlZelq6ZBPYdLuYPQlhS3dpAssjjmlMCcIGJyTGEIckUJuHLMBIsgHLlubuH9RTFMxDWPZY+Ps5Z86J515pfjMHfXN1Z0YjCSEEiIh6mUbtAoiof2L4EJEqGD5EpAqGDxGpguFDRKpg+BCRKhg+RKQKhg8RqYLhQ0SqYPgQkSpUC5/XXnsNQ4cOxYABA5CTk4Pq6mq1SiEiFagSPv/93/+NoqIivPjii/joo48watQo5OXlwel0qlEOEalAUuPG0pycHNx99934t3/7NwCALMsYPHgwFi1ahJ///Oe9XQ4RqUDX2xtsb29HTU0NiouLlXUajQa5ubmw2+1dvsbr9cLr9Sp/y7KM5uZmxMfHQ5KkHq+ZiG6OEAItLS1ISUmBRnP9L1a9Hj5nz56F3++HxWIJWG+xWPDZZ591+ZrS0lIsX768N8ojohBobGzEoEGDrtun18PnVhQXF6OoqEj52+12IzU1FY2NjTAajSpWRkRX8ng8GDx4MGJjY2/Yt9fDJyEhAVqtFk1NTQHrm5qaYLVau3yNwWCAwWC4ar3RaGT4EPVBNzMd0utnu/R6PcaOHYuKigplnSzLqKiogM1m6+1yiEglqnztKioqwty5c5GdnY1x48bhd7/7HS5cuIAnnnhCjXKISAWqhM/3v/99nDlzBiUlJXA4HBg9ejR27Nhx1SQ0Ed2+VLnOp7s8Hg9MJhPcbjfnfIj6kGA+m7y3i4hUwfAhIlUwfIhIFQwfIlIFw4eIVMHwISJVMHyISBUMHyJSBcOHiFTB8CEiVTB8iEgVDB8iUgXDh4hUwfAhIlUwfIhIFQwfIlIFw4eIVMHwISJVMHyISBUMHyJSBcOHiFTB8CEiVQQdPrt378ZDDz2ElJQUSJKEP//5zwHtQgiUlJQgOTkZkZGRyM3NxdGjRwP6NDc3Y/bs2TAajTCbzZg3bx5aW1u7tSNEFF6CDp8LFy5g1KhReO2117psX7lyJVavXo21a9eiqqoK0dHRyMvLQ1tbm9Jn9uzZqKurQ3l5ObZu3Yrdu3dj/vz5t74XRL3M6/XizJkzAYvX61W7rLDSrYcGSpKEzZs3Y8aMGQAuj3pSUlLwzDPP4NlnnwUAuN1uWCwWlJWV4fHHH8eRI0eQmZmJ/fv3Izs7GwCwY8cOPPDAAzhx4gRSUlJuuF0+NJB6myzLOHv2LKqra7F792c4e7YNJ060B/QZNEiPhAQD7rknA+PGZSExMREaTf+a2QjmsxnSxyUfO3YMDocDubm5yjqTyYScnBzY7XY8/vjjsNvtMJvNSvAAQG5uLjQaDaqqqvDoo49e9b5erzfg/yoejyeUZRNdU3t7O6qrP8HmzVU4cuQiLl0aiIiIBAASJEkK6HvihMCJEwL79x/DgAEfYfjwSMyYMQ45OaNhMBjU2YE+LKTh43A4AOCqZ65bLBalzeFwICkpKbAInQ5xcXFKn68rLS3F8uXLQ1kq0XUJIXDy5Gm88cYOfPjhJWi1iZAkDfT6a7/mchhJ0OvNkGUzDh2S8cknNbDZDmH+/GkYPDjlqsDqz8JiTFhcXAy3260sjY2NapdEtzEhBN5/346lS/+IvXsN0OmSIEnBf1QkSQOdLhH79g1AUdEm7NjxIWRZ7oGKw1NIRz5WqxUA0NTUhOTkZGV9U1MTRo8erfRxOp0Br/P5fGhublZe/3UGg4HDVuoVbW1t+NOf3seGDSfh86VAq+3+SEWr1cPjScHq1XVwOpvxT//0XURGRoag2vAW0pFPWloarFYrKioqlHUejwdVVVWw2WwAAJvNBpfLhZqaGqXPrl27IMsycnJyQlkOUVCEEFi/fivWrWuCzzcwpF+RJEmC3z8Q//VfZ/GHP7zLERBuYeTT2tqKL774Qvn72LFjOHjwIOLi4pCamoolS5bgV7/6FdLT05GWloYXXngBKSkpyhmx4cOHY9q0aXjyySexdu1adHR0YOHChXj88cdv6kwXUU8QQuC99+x4551z0GpDGzydJEmCVhuD//u/8/jmN/di2rTv9LuzYVcKOnwOHDiA+++/X/m7qKgIADB37lyUlZXhZz/7GS5cuID58+fD5XJh4sSJ2LFjBwYMGKC85q233sLChQsxZcoUaDQaFBQUYPXq1SHYHaJb87e/ncDrr1fB5/tGj04Kd46A1qzZj2HDBuNb3xrSY9vq67p1nY9aeJ0PhVJbWxtKSv4LH38cC40mpNOg1yTLfmRlufDyy3Nvq/mfYD6b/XfMR4TLX7d27dqHmhrRa8EDABqNFp98okF5+V6E4f//Q4LhQ/2az+fD//xPDbTagb2+ba3WjP/934Po6Ojo9W33BQwf6teOH29AQ8PVVyv3BkmScOKEBl9+ebzXt90XMHyo3xJCYM+eTyBEvIpVxOOvf/2kX371YvhQv+X3+/Hhh8eg0ag34avRDMDevX/rl1+9GD7Ub7W1taG52afq/VaSJMHl8gX85Ex/wfChfutvf2tEc/N17hTtJS5XJI4da1C7jF7H8KF+S5b7xjyLEOCcDxFRb2H4EJEqGD7UbyUmxiM62qd2GYiO7kBiYoLaZfQ6hg/1W3FxAxEb61d1vkUIgejoDiQkxKlWg1oYPtRvRUREYMwYK4RQb/QjhB+jRiUhIiJCtRrUwvChfkuSJNx330j4fOeCep0QMvx+L/x+L4Twd6sGn+8c7r8/q1/+rk/v3cZL1Ad9+9t3Ij6+HB6PuOHFhkL4cfHiSZw//yF8vhMAAJ0uGWbzBERFpUKj0Qa1bSEEBg68gKys4bdcfzjrf3FLdIXo6GhMnpwGWb7+Fcay7IPTWY7z59+EXn8c0dE+xMT4YTA0wuXaAKfz/+D3t1/3Pa5+Ty/uuy8VMTHR3dmFsMXwoX5NkiQUFNyHpCTXNSeehRA4e3Y3hPgIBoMEjeYfd8FLkoQBA7SQpEM4c6bipievhRBISGjGzJn398uvXADDhwhJSYn48Y/HQZK6nvtpb3ehra0GWu21f3pDo5HQ3v4p2trO3ORWm/HP/5wNq9Vy4663KYYP9XuSJOHee7Nx//1GyPKlq9pdrhoYDO3XnRO6PALyw+WqvuH2ZLkNkyZFY/LknH79EEGGDxEuPzV38eICjB59CbIceAaro6P5pr4aSZKEjo7m6/YRwo8RI1qxdOn3oNP17/M9DB+iv4uMHICFCx/EHXecCRgBSdLNn8W6Xl9ZbkNamhNPP/0goqJunx+Nv1UMH6IrpKam4Le//TEmT9bC7z8HIQRiY0fC673xQ/7a2/2IjR111XohBPz+Ztx7r4RVq36MoUMH9UTpYSeo8CktLcXdd9+N2NhYJCUlYcaMGaivrw/o09bWhsLCQsTHxyMmJgYFBQVoamoK6NPQ0ID8/HxERUUhKSkJy5Ytg8+n/j02RJIkITIyEkVF38PSpcNhNJ5CVNQgCPGN657JuhwwFhiNwwLW+/3tiI09hcWL78TPfjYTUVFR/Xqe50pBhU9lZSUKCwuxb98+lJeXo6OjA1OnTsWFCxeUPkuXLsWWLVuwadMmVFZW4tSpU3jssceUdr/fj/z8fLS3t2Pv3r1Yv349ysrKUFJSErq9Iuomg8GABx64F6tWzcTkyX5YLMNx8aLU5WOOZVmG1xuFhIQHoNFEQAgZHR1uaDTHMGmSF7/97ffw4IP3wWAwqLAnfVe3Hhp45swZJCUlobKyEvfeey/cbjcSExOxYcMGfO973wMAfPbZZxg+fDjsdjvGjx+P7du348EHH8SpU6dgsVw+zbh27Vo899xzOHPmDPT6G/+yHB8aSL1JlmU4nWewd+8+7Ny5DbW1tWhtbQUAtLW1AxiCuLh7ER0dCb2+A8OGReK++zKQkzMSFktSv7qOJ5jPZrem291uNwAgLu7yHbk1NTXo6OhAbm6u0icjIwOpqalK+NjtdmRlZSnBAwB5eXlYsGAB6urqMGbMmKu24/V64fV6A3aQqLdoNBpYrRY8+ujDmDHjIXz55Zc4dOgQgMvBpNVGQ5Ik3HlnGhIT4xAXFwdJUudxPOHklsNHlmUsWbIEEyZMwIgRIwAADocDer0eZrM5oK/FYoHD4VD6XBk8ne2dbV0pLS3F8uXLb7VUopDoDJT09HSkp6erXU7Yu+XxYGFhIQ4dOoSNGzeGsp4uFRcXw+12K0tjY2OPb5OIetYtjXwWLlyIrVu3Yvfu3Rg06B+nDa1WK9rb2+FyuQJGP01NTbBarUqf6urAq0A7z4Z19vk6g8HAyTqi20xQIx8hBBYuXIjNmzdj165dSEtLC2gfO3YsIiIiUFFRoayrr69HQ0MDbDYbAMBms6G2thZOp1PpU15eDqPRiMzMzO7sCxGFkaBGPoWFhdiwYQPeeecdxMbGKnM0JpMJkZGRMJlMmDdvHoqKihAXFwej0YhFixbBZrNh/PjxAICpU6ciMzMTc+bMwcqVK+FwOPD888+jsLCQoxui/kQEAUCXy7p165Q+ly5dEj/96U/FwIEDRVRUlHj00UfF6dOnA97n+PHjYvr06SIyMlIkJCSIZ555RnR0dNx0HW63WwAQbrc7mPKJqIcF89ns1nU+auF1PkR9UzCfzf5z9RMR9SkMHyJSBcOHiFTB8CEiVTB8iEgVDB8iUgXDh4hUwfAhIlUwfIhIFQwfIlIFw4eIVMHwISJVMHyISBUMHyJSBcOHiFTB8CEiVTB8iEgVDB8iUgXDh4hUwfAhIlUwfIhIFQwfIlJFUOGzZs0ajBw5EkajEUajETabDdu3b1fa29raUFhYiPj4eMTExKCgoEB5FHKnhoYG5OfnIyoqCklJSVi2bBl8Pl9o9oaIwkZQ4TNo0CCsWLECNTU1OHDgACZPnoxHHnkEdXV1AIClS5diy5Yt2LRpEyorK3Hq1Ck89thjyuv9fj/y8/PR3t6OvXv3Yv369SgrK0NJSUlo94qI+r7uPqFw4MCB4j/+4z+Ey+USERERYtOmTUrbkSNHBABht9uFEEJs27ZNaDQa4XA4lD5r1qwRRqNReL3em94mn1hK1DcF89m85Tkfv9+PjRs34sKFC7DZbKipqUFHRwdyc3OVPhkZGUhNTYXdbgcA2O12ZGVlwWKxKH3y8vLg8XiU0VNXvF4vPB5PwEJE4S3o8KmtrUVMTAwMBgOeeuopbN68GZmZmXA4HNDr9TCbzQH9LRYLHA4HAMDhcAQET2d7Z9u1lJaWwmQyKcvgwYODLZuI+pigw2fYsGE4ePAgqqqqsGDBAsydOxeHDx/uidoUxcXFcLvdytLY2Nij2yOinqcL9gV6vR533HEHAGDs2LHYv38//vVf/xXf//730d7eDpfLFTD6aWpqgtVqBQBYrVZUV1cHvF/n2bDOPl0xGAwwGAzBlkpEfVi3r/ORZRlerxdjx45FREQEKioqlLb6+no0NDTAZrMBAGw2G2pra+F0OpU+5eXlMBqNyMzM7G4pRBRGghr5FBcXY/r06UhNTUVLSws2bNiADz74ADt37oTJZMK8efNQVFSEuLg4GI1GLFq0CDabDePHjwcATJ06FZmZmZgzZw5WrlwJh8OB559/HoWFhRzZEPUzQYWP0+nED3/4Q5w+fRomkwkjR47Ezp078d3vfhcAsGrVKmg0GhQUFMDr9SIvLw+vv/668nqtVoutW7diwYIFsNlsiI6Oxty5c/HSSy+Fdq+IqM+ThBBC7SKC5fF4YDKZ4Ha7YTQa1S6HiP4umM8m7+0iIlUwfIhIFQwfIlIFw4eIVMHwISJVMHyISBUMHyJSBcOHiFTB8CEiVTB8iEgVDB8iUgXDh4hUwfAhIlUwfIhIFQwfIlIFw4eIVMHwISJVMHyISBUMHyJSBcOHiFTB8CEiVTB8iEgV3QqfFStWQJIkLFmyRFnX1taGwsJCxMfHIyYmBgUFBcojkTs1NDQgPz8fUVFRSEpKwrJly+Dz+bpTChGFmVsOn/379+Pf//3fMXLkyID1S5cuxZYtW7Bp0yZUVlbi1KlTeOyxx5R2v9+P/Px8tLe3Y+/evVi/fj3KyspQUlJy63tBROFH3IKWlhaRnp4uysvLxaRJk8TixYuFEEK4XC4REREhNm3apPQ9cuSIACDsdrsQQoht27YJjUYjHA6H0mfNmjXCaDQKr9d7U9t3u90CgHC73bdSPhH1kGA+m7c08iksLER+fj5yc3MD1tfU1KCjoyNgfUZGBlJTU2G32wEAdrsdWVlZsFgsSp+8vDx4PB7U1dV1uT2v1wuPxxOwEFF4C+pZ7QCwceNGfPTRR9i/f/9VbQ6HA3q9HmazOWC9xWKBw+FQ+lwZPJ3tnW1dKS0txfLly4MtlYj6sKBGPo2NjVi8eDHeeustDBgwoKdqukpxcTHcbreyNDY29tq2iahnBBU+NTU1cDqduOuuu6DT6aDT6VBZWYnVq1dDp9PBYrGgvb0dLpcr4HVNTU2wWq0AAKvVetXZr86/O/t8ncFggNFoDFiIKLwFFT5TpkxBbW0tDh48qCzZ2dmYPXu28t8RERGoqKhQXlNfX4+GhgbYbDYAgM1mQ21tLZxOp9KnvLwcRqMRmZmZIdotIurrgprziY2NxYgRIwLWRUdHIz4+Xlk/b948FBUVIS4uDkajEYsWLYLNZsP48eMBAFOnTkVmZibmzJmDlStXwuFw4Pnnn0dhYSEMBkOIdouI+rqgJ5xvZNWqVdBoNCgoKIDX60VeXh5ef/11pV2r1WLr1q1YsGABbDYboqOjMXfuXLz00kuhLoWI+jBJCCHULiJYHo8HJpMJbreb8z9EfUgwn03e20VEqmD4EJEqGD5EpAqGDxGpguFDRKpg+BCRKhg+RKQKhg8RqYLhQ0SqCPntFX2FEAIdHR2QZVlZp9FoEBERAUmSVKyMiIDbLHx8Ph+czjOorj6E+vpGHDnSDI+nXWmPiorAiBHx+Na3kpGTk4WUFCt0utvqEBCFjbC/tys2NhYXL17EX/5ShV27DuOzzy6hrc0MrTYGkqQNGOUIISCEH37/Rej153HnnXpMmjQM3/2uDbGxsRwREXVTMPd2hXX4uFwuHDt2Aq+++j6OH4+EVmuCJN38NJYQMvz+FiQnt2DhwkkYN240NBpOgxHdqn4TPr/5TRnee88DWU6EJGlv+f2EkAGcwbRpAzF//gzExESHrliifqTf3NW+ZcslyLKlW8ED4O+jpSRs3y7jlVf+iMbGkwjDTCYKK2EdPhpNZMjmaSRJgkYzANXVejz77B/hdJ4JyfsSUdfCOnx6glarR3NzIl59dTtaWy+oXQ7RbYvh0wWNxoB9+4A339wecJ0QEYUOw+caNJpIbN58ErW1n3H+h6gHMHyuQZIkCGHF6tXvoa2tTe1yiG47DJ/rkCQNjh0zwG7/SO1SiG47DJ8b0GrN+OCDOn71IgqxoMLnF7/4BSRJClgyMjKU9ra2NhQWFiI+Ph4xMTEoKCi46tHIDQ0NyM/PR1RUFJKSkrBs2TL4fL7Q7E0P0Gh0+PRTF86fP692KUS3laBHPt/+9rdx+vRpZdmzZ4/StnTpUmzZsgWbNm1CZWUlTp06hccee0xp9/v9yM/PR3t7O/bu3Yv169ejrKwMJSUlodmbHuLxGHHw4BG1yyC6rQR9S7dOp4PVar1qvdvtxh/+8Ads2LABkydPBgCsW7cOw4cPx759+zB+/Hi89957OHz4MN5//31YLBaMHj0av/zlL/Hcc8/hF7/4BfR6fff3KASu/oo1ACdPnoEQgjefEoVI0COfo0ePIiUlBd/85jcxe/ZsNDQ0AABqamrQ0dGB3NxcpW9GRgZSU1Nht9sBAHa7HVlZWbBYLEqfvLw8eDwe1NXVXXObXq8XHo8nYOkJl38D6ALOnv0QjY1laGwsg9NZAVnuQHX18R7ZJlF/FdTIJycnB2VlZRg2bBhOnz6N5cuX45577sGhQ4fgcDig1+thNpsDXmOxWOBwOAAADocjIHg62zvbrqW0tBTLly8PptSgCSHgdtfC49kFvf4CIiMv57Isn0JT00EkJ2dBlmVotd27j4yILgtq5DN9+nTMnDkTI0eORF5eHrZt2waXy4U//vGPPVUfAKC4uBhut1tZGhsbQ/r+Qgh4PHW4eHE7IiMvQafTKhPqWq0WUVE+fPnlp3j77bd51osoRLp1qt1sNuPOO+/EF198AavVivb2drhcroA+TU1NyhyR1Wq96uxX599dzSN1MhgMMBqNAUsoybIXbnclIiLka87p+P1+rFu37qr6iejWdCt8Wltb8eWXXyI5ORljx45FREQEKioqlPb6+no0NDTAZrMBAGw2G2pra+F0OpU+5eXlMBqNyMzM7E4p3eJ2H0JExPkbTiY3NzfjT3/6E0c/RCEQ1JzPs88+i4ceeghDhgzBqVOn8OKLL0Kr1WLWrFkwmUyYN28eioqKEBcXB6PRiEWLFsFms2H8+PEAgKlTpyIzMxNz5szBypUr4XA48Pzzz6OwsBAGg6FHdvBmXLrUCL3+xnM5Go0GBw8e7PmCiPqBoMLnxIkTmDVrFs6dO4fExERMnDgR+/btQ2JiIgBg1apV0Gg0KCgogNfrRV5eHl5//XXl9VqtFlu3bsWCBQtgs9kQHR2NuXPn4qWXXgrtXvUQjniIQiesf0Z18uQy6HRR3X4/l6sWFy++e8PRjyzLWLx4MebOncvrfYi60G9+RjVUYmOHwedLvO7IRggBi8WChx9+mMFDFAIMH1z+9cKBA6egvT2iywASQsBgMGDhwoUYOHCgChUS3X74xLy/i4n5JoAZOH++AjrdGUREXP4KJoTA0KFD8ZOf/ARTp07lqIcoRBg+fydJEmJj70BUVCo8ns9w4cLnsFi8KCycg6lTpyImJobBQxRCDJ+v0Wr1MJuzEBf3DRQXj8G9945j6BD1AM75dEGWz+MnP7kT99xzN4OHqIeEdfiE+iqBy+93DpMnR2L69O/w0clEPSisv3ZptU4IkdrtJ5YClx+ZLElOLFo0CtOmfQc6XVgfGqI+L6z/1/7KK7n41rfOQJZbb3kUJISALF/CN77hwL/8y/144IGJDB6iXhDWVzi7XC7o9Xps3LgT7777BdzuGGi1Zmg0Nx4JCeGHz+dBdLQH06en4oc/fADR0dGc4yHqhmCucA7r8OncQSEEzp07h5qaw6isPIza2ha0tOig1Q6EJEVc8UoffL7ziInpwPDh0Zg0KQPZ2d+GxZLE0CEKgX4XPleSZRnnzjWjqcmJjz8+CperVWmLiYlEdvYwJCYmIDExgb9KSBRiwYTPbTe5odFolHAZMUK93wgiousL6wlnIgpfDB8iUgXDh4hUwfAhIlUwfIhIFQwfIlIFw4eIVMHwISJVMHyISBVBh8/Jkyfxgx/8APHx8YiMjERWVhYOHDigtAshUFJSguTkZERGRiI3NxdHjx4NeI/m5mbMnj0bRqMRZrMZ8+bNQ2tr69c3RUS3saDC5/z585gwYQIiIiKwfft2HD58GL/5zW8CnuiwcuVKrF69GmvXrkVVVRWio6ORl5eHtrY2pc/s2bNRV1eH8vJybN26Fbt378b8+fNDt1dE1PeJIDz33HNi4sSJ12yXZVlYrVbx61//WlnncrmEwWAQb7/9thBCiMOHDwsAYv/+/Uqf7du3C0mSxMmTJ2+qDrfbLQAIt9sdTPlE1MOC+WwGNfJ59913kZ2djZkzZyIpKQljxozB73//e6X92LFjcDgcyM3NVdaZTCbk5OTAbrcDAOx2O8xmM7Kzs5U+ubm50Gg0qKqq6nK7Xq8XHo8nYCGi8BZU+Hz11VdYs2YN0tPTsXPnTixYsABPP/001q9fDwBwOBwAAIvFEvA6i8WitDkcDiQlJQW063Q6xMXFKX2+rrS0FCaTSVkGDx4cTNlE1AcFFT6yLOOuu+7CK6+8gjFjxmD+/Pl48sknsXbt2p6qDwBQXFwMt9utLI2NjT26PSLqeUGFT3JyMjIzA38jZ/jw4WhoaAAAWK1WAEBTU1NAn6amJqXNarXC6XQGtPt8PjQ3Nyt9vs5gMMBoNAYsRBTeggqfCRMmoL6+PmDd559/jiFDhgAA0tLSYLVaUVFRobR7PB5UVVXBZrMBAGw2G1wuF2pqapQ+u3btgizLyMnJueUdIaIwE8xMdnV1tdDpdOLll18WR48eFW+99ZaIiooSb775ptJnxYoVwmw2i3feeUd8+umn4pFHHhFpaWni0qVLSp9p06aJMWPGiKqqKrFnzx6Rnp4uZs2a1SMz6kTUe4L5bAYVPkIIsWXLFjFixAhhMBhERkaGeOONNwLaZVkWL7zwgrBYLMJgMIgpU6aI+vr6gD7nzp0Ts2bNEjExMcJoNIonnnhCtLS03HQNDB+ivimYz+Zt9wPyRKSeYD6bvLeLiFTB8CEiVTB8iEgVDB8iUgXDh4hUwfAhIlUwfIhIFQwfIlIFw4eIVMHwISJVMHyISBUMHyJSBcOHiFTB8CEiVTB8iEgVDB8iUgXDh4hUwfAhIlUwfIhIFQwfIlIFw4eIVMHwISJVMHyISBUMHyJSBcOHiFShU7uAW9H5kFWPx6NyJUR0pc7P5M08CDksw+fcuXMAgMGDB6tcCRF1paWlBSaT6bp9wjJ84uLiAAANDQ033MHbmcfjweDBg9HY2Nivn1nP43BZXzgOQgi0tLQgJSXlhn3DMnw0mstTVSaTqV//Y+tkNBp5HMDj0Ent43CzAwJOOBORKhg+RKSKsAwfg8GAF198EQaDQe1SVMXjcBmPw2XhdhwkcTPnxIiIQiwsRz5EFP4YPkSkCoYPEamC4UNEqgjL8HnttdcwdOhQDBgwADk5Oaiurla7pJApLS3F3XffjdjYWCQlJWHGjBmor68P6NPW1obCwkLEx8cjJiYGBQUFaGpqCujT0NCA/Px8REVFISkpCcuWLYPP5+vNXQmpFStWQJIkLFmyRFnXX47DyZMn8YMf/ADx8fGIjIxEVlYWDhw4oLQLIVBSUoLk5GRERkYiNzcXR48eDXiP5uZmzJ49G0ajEWazGfPmzUNra2tv70ogEWY2btwo9Hq9+M///E9RV1cnnnzySWE2m0VTU5PapYVEXl6eWLdunTh06JA4ePCgeOCBB0RqaqpobW1V+jz11FNi8ODBoqKiQhw4cECMHz9efOc731HafT6fGDFihMjNzRUff/yx2LZtm0hISBDFxcVq7FK3VVdXi6FDh4qRI0eKxYsXK+v7w3Fobm4WQ4YMET/60Y9EVVWV+Oqrr8TOnTvFF198ofRZsWKFMJlM4s9//rP45JNPxMMPPyzS0tLEpUuXlD7Tpk0To0aNEvv27RN//etfxR133CFmzZqlxi4pwi58xo0bJwoLC5W//X6/SElJEaWlpSpW1XOcTqcAICorK4UQQrhcLhERESE2bdqk9Dly5IgAIOx2uxBCiG3btgmNRiMcDofSZ82aNcJoNAqv19u7O9BNLS0tIj09XZSXl4tJkyYp4dNfjsNzzz0nJk6ceM12WZaF1WoVv/71r5V1LpdLGAwG8fbbbwshhDh8+LAAIPbv36/02b59u5AkSZw8ebLnir+BsPra1d7ejpqaGuTm5irrNBoNcnNzYbfbVays57jdbgD/uJm2pqYGHR0dAccgIyMDqampyjGw2+3IysqCxWJR+uTl5cHj8aCurq4Xq+++wsJC5OfnB+wv0H+Ow7vvvovs7GzMnDkTSUlJGDNmDH7/+98r7ceOHYPD4Qg4DiaTCTk5OQHHwWw2Izs7W+mTm5sLjUaDqqqq3tuZrwmr8Dl79iz8fn/APyYAsFgscDgcKlXVc2RZxpIlSzBhwgSMGDECAOBwOKDX62E2mwP6XnkMHA5Hl8eosy1cbNy4ER999BFKS0uvausvx+Grr77CmjVrkJ6ejp07d2LBggV4+umnsX79egD/2I/rfSYcDgeSkpIC2nU6HeLi4lQ9DmF5V3t/UVhYiEOHDmHPnj1ql9LrGhsbsXjxYpSXl2PAgAFql6MaWZaRnZ2NV155BQAwZswYHDp0CGvXrsXcuXNVrq57wmrkk5CQAK1We9UZjaamJlitVpWq6hkLFy7E1q1b8Ze//AWDBg1S1lutVrS3t8PlcgX0v/IYWK3WLo9RZ1s4qKmpgdPpxF133QWdTgedTofKykqsXr0aOp0OFoulXxyH5ORkZGZmBqwbPnw4GhoaAPxjP673mbBarXA6nQHtPp8Pzc3Nqh6HsAofvV6PsWPHoqKiQlknyzIqKipgs9lUrCx0hBBYuHAhNm/ejF27diEtLS2gfezYsYiIiAg4BvX19WhoaFCOgc1mQ21tbcA/uPLychiNxqv+IfdVU6ZMQW1tLQ4ePKgs2dnZmD17tvLf/eE4TJgw4apLLT7//HMMGTIEAJCWlgar1RpwHDweD6qqqgKOg8vlQk1NjdJn165dkGUZOTk5vbAX16DaVPct2rhxozAYDKKsrEwcPnxYzJ8/X5jN5oAzGuFswYIFwmQyiQ8++ECcPn1aWS5evKj0eeqpp0RqaqrYtWuXOHDggLDZbMJmsyntnaeYp06dKg4ePCh27NghEhMTw+oUc1euPNslRP84DtXV1UKn04mXX35ZHD16VLz11lsiKipKvPnmm0qfFStWCLPZLN555x3x6aefikceeaTLU+1jxowRVVVVYs+ePSI9PZ2n2m/Fq6++KlJTU4Verxfjxo0T+/btU7ukkAHQ5bJu3Tqlz6VLl8RPf/pTMXDgQBEVFSUeffRRcfr06YD3OX78uJg+fbqIjIwUCQkJ4plnnhEdHR29vDeh9fXw6S/HYcuWLWLEiBHCYDCIjIwM8cYbbwS0y7IsXnjhBWGxWITBYBBTpkwR9fX1AX3OnTsnZs2aJWJiYoTRaBRPPPGEaGlp6c3duAp/UoOIVBFWcz5EdPtg+BCRKhg+RKQKhg8RqYLhQ0SqYPgQkSoYPkSkCoYPEamC4UNEqmD4EJEqGD5EpAqGDxGp4v8BZ5WIiDDHYjkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-86.06948852539062"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第9章-策略梯度算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
